# -*- coding: utf-8 -*-
"""
Created on Fri Jan 14 12:52:15 2022

@author: 胡自成
"""

# ============导入库============

from glob import glob
from logging import exception
from bilibili_api import video, sync, comment, user # 导入bilibili_api库的一些功能
from bilibili_api.utils.aid_bvid_transformer import bvid2aid as ba # 导入bvid转换为avid的函数
from bilibili_api.utils.Credential import Credential
from bilibili_api import settings

#import nest_asyncio # 异步IO初始化
#nest_asyncio.apply()
import requests
import re
import json
import pprint as pp
import random
import time
from bs4 import BeautifulSoup
from openpyxl import Workbook 
from openpyxl import load_workbook
import os
import datetime
import sys

from fake_useragent import UserAgent
import json

# ============初始化全局变量============
def get_bili_cfg():
    json_file = "cfg.json"
    with open(json_file) as f:
        cfg = json.load(f)
    return cfg

cfg = get_bili_cfg()

debug_lv_no    = 0 # 没有调试信息
debug_lv_final = 1 # 调试最终输出
debug_lv_all   = 2 # 调试过程输出

debug_level    = cfg["debug_level"] # default level

total_seconds = 0  # 总共耗时

enable_try_catch = False # 打开尝试获取特性（失败后重新开始）
if cfg["try_catch"] == "yes":
    enable_try_catch = True

ip_pool_file = "ip_pool.txt"

random_proxy_ip = False
if cfg["random_proxy"] == "yes":
    random_proxy_ip = True

proxy_ip_timeslice = 0
proxy_ip_timeslice_max = cfg["proxy_timeslice"]

# less data means only get video info
less_data = False
if cfg["less_data"] == "yes":
    less_data = True

try_get_video_count = cfg["try_get_video_count"] # 单个视频失败后尝试次数

cookie_sessdata = cfg["cookie_sessdata"]
cookie_bili_jct = cfg["cookie_bili_jct"]

myCredential = Credential(cookie_sessdata, cookie_bili_jct)

# 数据抓取的时候容易失败，因此这里设置一些全局变量，保存已经获取得评论，减少多次获取失败。
comments_backup = []
comment_page = 1
comment_count = 0

video_info_backup = {}
danmakus_backup = []
user_info_backup = {}
video_pub_time_backup = ""
user_archive_count_backup = 0

# ============定义函数============
def make_cookies():
    global cookie_sessdata, cookie_bili_jct
    cookies = {"SESSDATA":cookie_sessdata, "bili_jct":cookie_bili_jct}
    return cookies

def get_ip_proxy(path):

    if not os.path.exists(path):
        with open(path, "w") as fp: # 创建文件
            pass

    with open(path, "r") as fp:
        proxy_ip_list_raw = fp.readlines()

    proxy_ip_list = []
    for ip in proxy_ip_list_raw:
        proxy_ip_list.append(ip.strip("\n"))

    # add no proxy
    proxy_ip_list.append("localhost")
    return proxy_ip_list

def update_proxy_ip(force=False):
    global ip_pool_file, random_proxy_ip, proxy_ip_timeslice, proxy_ip_timeslice_max
    if random_proxy_ip:
        proxy_ip_timeslice += 1
        if proxy_ip_timeslice > proxy_ip_timeslice_max or force:
            proxy_ip_timeslice = 0
            proxy_ip_list = get_ip_proxy(ip_pool_file)
            proxy = random.choice(proxy_ip_list)
            if proxy == "localhost":
                settings.proxy = ""
            else:
                settings.proxy = "http://" + proxy
            print("[INFO] set proxy as ", settings.proxy)

def do_exit():
    import sys
    sys.exit()
    
def do_sleep():
    second = random.randint(1, 2)
    print(f".... 随机休眠: {second} 秒，避免太快了被反爬封IP地址...")
    time.sleep(second)

# aid对应视频av号，通过视频av号获取oid
def get_cid_by_bvid(bvid):
    global debug_level, debug_lv_all
    
    request_headers = {'user-agent': UserAgent().random}
    
    url = "https://api.bilibili.com/x/player/pagelist?bvid=" + str(bvid) + "&jsonp=jsonp"
    if debug_level >= debug_lv_all:
        print("API URL for cid:", url)
    
    response = requests.get(url, headers=request_headers, cookies=make_cookies())
    dirt = json.loads(response.text)
    cid = dirt['data'][0]['cid']
    return cid

def get_danmakus(bvid):
    global debug_level, debug_lv_all, less_data
    global danmakus_backup

    if less_data:
        return []

    if danmakus_backup:
        return danmakus_backup

    request_headers = {'user-agent': UserAgent().random}

    # 通过bvid获取对应的oid，然后再通过oid获取弹幕内容
    oid = get_cid_by_bvid(bvid)

    url = 'https://api.bilibili.com/x/v1/dm/list.so?oid=' + str(oid)
    if debug_level >= debug_lv_all:
        print("API URL for 弹幕:", url)
    
    response = requests.get(url=url, headers=request_headers, cookies=make_cookies())
    # err response
    if response == None:
        return []

    # default encoding is utf-8
    try:
        response.encoding = response.apparent_encoding
    except:
        response.encoding = "utf-8"
    
    # re 正则表达式
    html_data = re.findall('<d p=".*?">(.*?)</d>', response.text)
    
    danmakus_backup = html_data

    # 返回弹幕
    return html_data

async def get_comment(bvid):
    global debug_level, debug_lv_all, less_data, myCredential
    global comments_backup, comment_page, comment_count

    if less_data:
        return []

    while True:
        print("[INFO] 获取评论页面：", comment_page)
        print("[INFO] 已经获取的评论数量：", comment_count)

        # 获取评论
        avid = ba(bvid) # bvid转换成avid

        c = await comment.get_comments(avid, comment.ResourceType.VIDEO, comment_page, credential=myCredential)
        # 存储评论
        comments_backup.extend(c['replies'])
        # 增加已获取数量
        comment_count += c['page']['size']
        # 增加页码
        comment_page += 1

        if comment_count >= c['page']['count']:
            # 当前已获取数量已达到评论总数，跳出循环
            break
    
    result = []
    # 打印评论
    for cmt in comments_backup:    
        result.append([cmt['member']['uname'], cmt['content']['message']])
    
        if debug_level >= debug_lv_all:
            print(f"{cmt['member']['uname']}: {cmt['content']['message']}")
        
    # 返回评论
    return result

async def get_video_info(bvid):
    """    
    获取一个视频的信息
    """
    global debug_level, debug_lv_all, myCredential
    global video_info_backup

    if video_info_backup:
        return video_info_backup

    v = video.Video(bvid=bvid, credential=myCredential)
    # 获取信息
    info = await v.get_info()
    # 打印信息
    if debug_level >= debug_lv_all:
        print("原始视频信息：", info)
        print("人性化视频信息：")
        pp.pprint(info, width=30)

    
    result = {}
    result['标题'] = info['title']
    result['bvid'] = bvid
    result['aid'] = info['stat']['aid']
    result['播放数量'] = info['stat']['view']
    result['投币数量'] = info['stat']['coin']
    result['点赞数量'] = info['stat']['like']
    result['点踩数量'] = info['stat']['dislike']
    result['收藏数量'] = info['stat']['favorite']
    result['弹幕数量'] = info['stat']['danmaku']
    result['分享数量'] = info['stat']['share']
    result['评论数量'] = info['stat']['reply']
    result['视频时长'] = info['duration']
    result['用户id'] = info['owner']['mid']
    result['用户名'] = info['owner']['name']
    
    video_info_backup = result

    # 返回我们需要的信息
    return result
    
def get_time(bvid):
    global video_pub_time_backup

    if video_pub_time_backup:
        return video_pub_time_backup

    url = "https://www.bilibili.com/video/" + bvid
    # 日期在class为video-data的标签里面
    my_attrs = {'class' : 'video-data'}

    request_headers = {'user-agent': UserAgent().random}
    response = requests.get(url, headers=request_headers, cookies=make_cookies())
    if response == None:
        return None
    soup = BeautifulSoup(response.text,'lxml')
    # 查找数据所在的块
    div = soup.find(name='div',attrs=my_attrs)
    
    # 重新查找一个span
    div_soup = BeautifulSoup(str(div),'lxml')
    div = div_soup.find_all(name='span')
    
    # 第3个span是发布时间
    time = div[2].text

    video_pub_time_backup = time

    return time

def get_user_info(uid):
    global user_info_backup

    if user_info_backup:
        return user_info_backup

    print("[INFO] user id:", uid)
    u = user.User(uid)
    uinfo_list = {}
    uinfo = sync(u.get_relation_info())
    
    uinfo_list["关注数"] = uinfo["following"]
    uinfo_list["粉丝数"] = uinfo["follower"]

    uinfo = sync(u.get_user_info())    
    uinfo_list["等级"] = uinfo["level"]    
    
    user_info_backup = uinfo_list

    return uinfo_list

def get_archive_count(uid):
    global user_archive_count_backup

    if user_archive_count_backup:
        return user_archive_count_backup

    print("[INFO] user id:", uid)
    url = 'https://api.bilibili.com/x/web-interface/card?mid=' + str(uid)
    request_headers = {'user-agent': UserAgent().random}
    wb_data = requests.get(url, headers=request_headers, cookies=make_cookies())
    if wb_data == None:
        return None
    json_data = json.loads(wb_data.text)
    if json_data["code"] != 0:
        return None

    user_archive_count_backup = json_data["data"]["archive_count"]

    return json_data["data"]["archive_count"]


def get_one_video(bvid):
    update_proxy_ip()
    print("[INFO] get video info")
    # 获取视频的基础信息
    basic_res = sync(get_video_info(bvid))
    print("[INFO] get user info")
    
    # 获取用户的一些信息
    user_info = get_user_info(basic_res['用户id'])
    
    basic_res['关注数'] = user_info['关注数']
    basic_res['粉丝数'] = user_info['粉丝数']
    basic_res['等级'] = user_info['等级']

    print("[INFO] get archive count")
    basic_res['视频数量'] = get_archive_count(basic_res['用户id'])
    if basic_res['视频数量'] == None:
        return None
    if debug_level >= debug_lv_final:
        print("---> 基础信息：", basic_res)
    print(f"[INFO] 标题：{basic_res['标题']}")
    print(f"[INFO] 播放数量：{basic_res['播放数量']}")
    print(f"[INFO] 投币数量：{basic_res['投币数量']}")
    print(f"[INFO] 点赞数量：{basic_res['点赞数量']}")
    print(f"[INFO] 收藏数量：{basic_res['收藏数量']}")
    print(f"[INFO] 弹幕数量：{basic_res['弹幕数量']}")
    print(f"[INFO] 分享数量：{basic_res['分享数量']}")
    print(f"[INFO] 评论数量：{basic_res['评论数量']}")
    print(f"[INFO] 视频时长：{basic_res['视频时长']} 秒")
    print(f"[INFO] 用户id：{basic_res['用户id']}")
    print(f"[INFO] 用户名：{basic_res['用户名']}")
    print(f"[INFO] 关注数：{basic_res['关注数']}")
    print(f"[INFO] 粉丝数：{basic_res['粉丝数']}")
    print(f"[INFO] 等级：{basic_res['等级']}")
    print(f"[INFO] 视频数量：{basic_res['视频数量']}")

    print("[INFO] get time")
    # 获取时间
    time_res = get_time(bvid)
    if time_res == None:
        return None
    if debug_level >= debug_lv_final:
        print("[INFO] 发布日期：", time_res)
    print(f"[INFO] 发布日期：{time_res}")

    update_proxy_ip()
    # 获取弹幕
    print("[INFO] 开始抓取弹幕")
    danmakus_res = get_danmakus(bvid)
    print(f"[INFO] 共有：{len(danmakus_res)} 条弹幕")
    if debug_level >= debug_lv_final:
        print("---> 弹幕：", danmakus_res)
    
    update_proxy_ip()

    # 获取评论
    print("[INFO] 开始抓取评论")
    comment_res = sync(get_comment(bvid))
    print(f"[INFO] 共有：{len(comment_res)} 条评论（不含子评论）")
    if debug_level >= debug_lv_final:
        print("[INFO] 评论：", comment_res)
    
    # 整个视频信息到一个变量中
    video_res = basic_res
    video_res['URL'] = "https://www.bilibili.com/video/" + bvid
    video_res['发布日期'] = time_res
    video_res['评论'] = comment_res
    video_res['弹幕'] = danmakus_res
    
    return video_res

def try_get_one_video(bvid):
    global try_get_video_count, comments_backup, comment_page, comment_count
    global video_info_backup, danmakus_backup, user_info_backup
    global video_pub_time_backup, user_archive_count_backup

    # 初始化视频的备份信息
    comments_backup = []
    comment_page = 1
    comment_count = 0

    video_info_backup = {}
    danmakus_backup = []
    user_info_backup = {}

    video_pub_time_backup = ""

    user_archive_count_backup = 0

    for i in range(try_get_video_count):
        try:
            result = get_one_video(bvid)
            if result == None:
                print("[INFO] try get one video return none!")
                update_proxy_ip(True)
            else:
                return result
        except Exception as e:
            print("[INFO] try get one video failed!:", e)
            update_proxy_ip(True)
    return None

def save_data(path, res):
    # 创建评论表格 
    wb = Workbook()
    
    # 重名Sheet为评论
    for ws in wb:
        ws.title = "评论"
        
    ws = wb.active
    
    ws.append(["用户", "评论"])
    for cmt in res["评论"]:
        try:
            ws.append([cmt[0], cmt[1]])
        except Exception as e:
            print("[ERROR] 解析评论数据异常：", e)
            print("[ERROR] 待解析的数据为：", cmt[0], cmt[1])

    ws = wb.create_sheet("弹幕")
    
    ws.append(["弹幕"])
    for danmu in res["弹幕"]:
        ws.append([danmu])
    
    wb.save(path + ".xlsx")

def save_info(path, res):
    full_path = path + "/info.xlsx"

    # 不存在则创建信息表格
    if not os.path.exists(full_path):
        wb = Workbook()
        ws = wb.active
        
        ws.append(["视频BVID", "视频AID", "标题", "发布日期", "播放数量", "投币数量",
                    "点赞数", "点踩数", "收藏数", "分享数", "评论数", "弹幕数",
                    "视频时长", "用户id", "用户名", "关注数", "粉丝数", "等级", "视频数量"])
        
    else: # 已经存在则直接打开
        wb = load_workbook(full_path)
        ws = wb.active
    
    # 追加数据
    info = [res['bvid'], res['aid'], res['标题'], res['发布日期'], res['播放数量'], res['投币数量'], 
            res['点赞数量'], res['点踩数量'], res['收藏数量'],  res['分享数量'],  res['评论数量'],
            res['弹幕数量'], res["视频时长"], res['用户id'], res["用户名"], res["关注数"], res["粉丝数"],
            res["等级"], res["视频数量"]]
    ws.append(info)
    
    # 修改后保存
    wb.save(full_path)

def save_to_excel(path, res, bvid):
    if not os.path.exists(path + "/videos"):
        os.mkdir(path + "/videos")
    save_data(path + "/videos/" + bvid, res)
    save_info(path, res)
    
def get_exist_videos(block_name):
    bvid_list = []
    
    info_path = "blocks/" + block_name + "/info.xlsx"

    if not os.path.exists(info_path):
        return bvid_list
    
    wb = load_workbook(info_path)
    ws = wb.active
    
    for i in range(2, ws.max_row + 1):
        _cell = ws.cell(row=i, column=1)    
        bvid_list.append(_cell.value)
    return bvid_list

def get_url_table(path):
    print(f"加载url文件：{path}")
    wb = load_workbook(path)
    ws = wb.active

    url_table = []

    for i in range(2, ws.max_row + 1):
        url = ws.cell(i, 2).value
        url_table.append(url)
    
    return url_table

def get_urls(path):
    filelist = os.listdir(path)
    url_table = []
    for filename in filelist:
        if filename.endswith(".xlsx"):
            url_table += get_url_table(path + "/" + filename)
    return url_table

def get_target_videos(block_name):
    bvid_list = []
    
    target_path = "blocks/" + block_name + "/targets"

    urls = get_urls(target_path)
    
    for url in urls:
        bvid_list.append(url[url.rfind("/") + 1:])      

    return bvid_list

def fetch_video(block_name):
    global total_seconds

    # 加载已经获取的视频
    exist_bvid_list = get_exist_videos(block_name)
    #print("#### 已存在视频：", exist_bvid_list)
        
    # 加载待获取的视频
    bvid_list = get_target_videos(block_name)
    #print("#### 抓取目标视频：", bvid_list)

    # 循环获取所有视频
    counts = len(bvid_list)
    idx = 1
    for bvid in bvid_list:
        if bvid in exist_bvid_list:
            idx += 1
            continue
        
        starttime = datetime.datetime.now()

        # 获取视频信息
        print(f"-> [{idx}/{counts}] 抓取视频: {bvid} 信息中...")
        try:
            res = try_get_one_video(bvid)
        except Exception as e:
            print("[INFO] one video exception:", e)
            idx += 1
            update_proxy_ip(True)
            continue

        if res == None:
            print(f"->  [{idx}/{counts}] 获取视频失败，处理下一次视频.")
            idx += 1
            continue

        print(f"->  [{idx}/{counts}] 保存视频: {bvid} 信息中...")
        
        # 储存到本地文件中
        save_path = "blocks/" + block_name
        save_to_excel(save_path, res, res['bvid'])
        print(f"->  [{idx}/{counts}] 抓取并保存视频: {bvid} 信息成功...")
        
        # 计算视频结束时间
        endtime = datetime.datetime.now()
        sec = (endtime - starttime).seconds

        print(f"-> 本次视频耗时：{int(sec / 60)}分:{sec % 60}秒")

        idx += 1
        # 执行休眠，避免操作太快导致封禁IP
        if idx <= counts:
            do_sleep()

        # 追加延时时间
        endtime = datetime.datetime.now()
        sec = (endtime - starttime).seconds
        total_seconds += sec

def try_fetch_video(n, block_name):
    if enable_try_catch:
        try:
            fetch_video(block_name)
            print("! 执行完毕！抓取结束！")
            return 0
        except Exception as e:
            print(f"\n!!! [{n}] 执行出错，再次尝试执行！！！")
            print(f"异常信息：", e, "\n")
        return n - 1
    else:
        fetch_video(block_name)
        print("! 执行完毕！抓取结束！")
        return 0
# ============主体框架============
def main():
    global total_seconds

    block_name = "block0" # default block0

    print("[INFO] 参数个数：", len(sys.argv))

    # 选择块名
    if len(sys.argv) >= 2:
        block_name = sys.argv[1]
        print("[INFO] set block as ", block_name)

    # 检查块是否存在
    if not os.path.exists("blocks/" + block_name):
        print(f"[ERR ] 操作块`{block_name}`不存在！")
        return

    # 多次尝试执行
    max_try_count = cfg["try_catch_count"]

    print("[INFO] 操作块名字：", block_name)
    print("[INFO] 最大尝试出错次数：", max_try_count)
    while max_try_count > 0:
        max_try_count = try_fetch_video(max_try_count, block_name)
    print(f"===> 总耗时：{int(total_seconds / 60)}分:{total_seconds % 60}秒")

if __name__ == '__main__':
    try:
        main()
    except Exception as e:
        print("[INFO] exception:", e)

    time.sleep(5)
